{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from browser import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = [\n",
    "    'C100_DenseBest_F',\n",
    "    'C100_DenseBest_Linear1_F',\n",
    "    'C100_SparseBest_F',\n",
    "    'C100_SparseBest_Linear1_F',\n",
    "]\n",
    "\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40, 62)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean_and_std</th>\n",
       "      <th>mean_and_std</th>\n",
       "      <th>mean_and_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_DenseBest_F</th>\n",
       "      <td>0.7257 ± 0.0023</td>\n",
       "      <td>0.4635 ± 0.0042</td>\n",
       "      <td>0.2190 ± 0.0078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_DenseBest_Linear1_F</th>\n",
       "      <td>0.7248 ± 0.0026</td>\n",
       "      <td>0.4568 ± 0.0049</td>\n",
       "      <td>0.2046 ± 0.0089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest_F</th>\n",
       "      <td>0.6985 ± 0.0019</td>\n",
       "      <td>0.4645 ± 0.0031</td>\n",
       "      <td>0.2607 ± 0.0084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest_Linear1_F</th>\n",
       "      <td>0.6964 ± 0.0029</td>\n",
       "      <td>0.4634 ± 0.0032</td>\n",
       "      <td>0.2676 ± 0.0119</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   test_accuracy_max mean_accuracy_max  \\\n",
       "                                        mean_and_std      mean_and_std   \n",
       "dataset  name                                                            \n",
       "CIFAR100 C100_DenseBest_F            0.7257 ± 0.0023   0.4635 ± 0.0042   \n",
       "         C100_DenseBest_Linear1_F    0.7248 ± 0.0026   0.4568 ± 0.0049   \n",
       "         C100_SparseBest_F           0.6985 ± 0.0019   0.4645 ± 0.0031   \n",
       "         C100_SparseBest_Linear1_F   0.6964 ± 0.0029   0.4634 ± 0.0032   \n",
       "\n",
       "                                   noise_accuracy_max  \n",
       "                                         mean_and_std  \n",
       "dataset  name                                          \n",
       "CIFAR100 C100_DenseBest_F             0.2190 ± 0.0078  \n",
       "         C100_DenseBest_Linear1_F     0.2046 ± 0.0089  \n",
       "         C100_SparseBest_F            0.2607 ± 0.0084  \n",
       "         C100_SparseBest_Linear1_F    0.2676 ± 0.0119  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def mean_and_std(s):\n",
    "    return \"{:.4f} ± {:.4f}\".format(s.mean(), s.std())\n",
    "\n",
    "(df.groupby(['dataset', 'name'])\n",
    "     .agg({'test_accuracy_max': [mean_and_std], \n",
    "           'mean_accuracy_max': [mean_and_std],\n",
    "           'noise_accuracy_max': [mean_and_std]}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">test_accuracy_max</th>\n",
       "      <th colspan=\"3\" halign=\"left\">mean_accuracy_max</th>\n",
       "      <th colspan=\"3\" halign=\"left\">noise_accuracy_max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_DenseBest_F</th>\n",
       "      <td>0.7296</td>\n",
       "      <td>0.7257</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.4709</td>\n",
       "      <td>0.4635</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.2371</td>\n",
       "      <td>0.2190</td>\n",
       "      <td>0.0078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_DenseBest_Linear1_F</th>\n",
       "      <td>0.7294</td>\n",
       "      <td>0.7248</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.4642</td>\n",
       "      <td>0.4568</td>\n",
       "      <td>0.0049</td>\n",
       "      <td>0.2206</td>\n",
       "      <td>0.2046</td>\n",
       "      <td>0.0089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest_F</th>\n",
       "      <td>0.7013</td>\n",
       "      <td>0.6985</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>0.4682</td>\n",
       "      <td>0.4645</td>\n",
       "      <td>0.0031</td>\n",
       "      <td>0.2740</td>\n",
       "      <td>0.2607</td>\n",
       "      <td>0.0084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest_Linear1_F</th>\n",
       "      <td>0.7020</td>\n",
       "      <td>0.6964</td>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.4684</td>\n",
       "      <td>0.4634</td>\n",
       "      <td>0.0032</td>\n",
       "      <td>0.2830</td>\n",
       "      <td>0.2676</td>\n",
       "      <td>0.0119</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   test_accuracy_max                  \\\n",
       "                                                 max    mean     std   \n",
       "dataset  name                                                          \n",
       "CIFAR100 C100_DenseBest_F                     0.7296  0.7257  0.0023   \n",
       "         C100_DenseBest_Linear1_F             0.7294  0.7248  0.0026   \n",
       "         C100_SparseBest_F                    0.7013  0.6985  0.0019   \n",
       "         C100_SparseBest_Linear1_F            0.7020  0.6964  0.0029   \n",
       "\n",
       "                                   mean_accuracy_max                  \\\n",
       "                                                 max    mean     std   \n",
       "dataset  name                                                          \n",
       "CIFAR100 C100_DenseBest_F                     0.4709  0.4635  0.0042   \n",
       "         C100_DenseBest_Linear1_F             0.4642  0.4568  0.0049   \n",
       "         C100_SparseBest_F                    0.4682  0.4645  0.0031   \n",
       "         C100_SparseBest_Linear1_F            0.4684  0.4634  0.0032   \n",
       "\n",
       "                                   noise_accuracy_max                  \n",
       "                                                  max    mean     std  \n",
       "dataset  name                                                          \n",
       "CIFAR100 C100_DenseBest_F                      0.2371  0.2190  0.0078  \n",
       "         C100_DenseBest_Linear1_F              0.2206  0.2046  0.0089  \n",
       "         C100_SparseBest_F                     0.2740  0.2607  0.0084  \n",
       "         C100_SparseBest_Linear1_F             0.2830  0.2676  0.0119  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.groupby(['dataset', 'name'])\n",
    "     .agg({'test_accuracy_max': ['max','mean', 'std'], \n",
    "           'mean_accuracy_max': ['max', 'mean', 'std'],\n",
    "           'noise_accuracy_max': ['max', 'mean', 'std']})\n",
    "     .round(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
